# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
import re,base64,json
from douban_movie.items import JobItem
from scrapy.exceptions import CloseSpider
from selenium import webdriver
from douban_movie.middlewares import JobDownloaderMiddleware as jdm


class ZhiLianSpider(scrapy.spiders.CrawlSpider):
    name = 'zhilian-web'
    dr = re.compile(r'<[^>]+>',re.S)
    @property
    def start_urls(self):
        url_tmpl = 'https://fe-api.zhaopin.com/c/i/sou?start={}&pageSize=60&cityId=538&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=Web%E5%89%8D%E7%AB%AF&kt=3&lastUrlQuery=%7B%22p%22:10,%22pageSize%22:%2260%22,%22jl%22:%22538%22,%22kw%22:%22Web%E5%89%8D%E7%AB%AF%22,%22kt%22:%223%22%7D'
        return (url_tmpl.format(i) for i in range(1, 60, 60))


    def parse(self, response):
        dd = self.dr.sub('',str(response.body, encoding = "utf8"))
        all = json.loads(dd)
        for job in all['data']['results']:
            item = JobItem({
                'name': job['jobName'],
                'job_saray': job['salary'],
                'exper': job['workingExp']['name'],
                'educ': job['eduLevel']['name'],
                'company': job['company']['name'],
                'welfare': ','.join(job['welfare'])
            })
            course_url = job['positionURL']
            request = scrapy.Request(course_url, callback=self.parse_other)
            request.meta['item'] = item
            yield request

    def parse_other(self, response):
        item = response.meta['item']
        # item['address'] = response.css('p.add-txt::text').extract_first()
        # item['responsibility'] = ''.join(response.css('div.responsibility div.pos-ul p::text').extract())
        # item['companyInfo'] = ','.join(response.css('ul.promulgator-ul li strong::text').extract())
        item['address'] = response.css('div.terminalpage-main div:nth-child(1) h2::text').extract_first()
        item['responsibility'] = response.css('div.terminalpage-main div:nth-child(1)').extract_first()
        item['companyInfo'] = response.css('ul.terminal-company').extract_first()
        yield item
